from elasticsearch import Elasticsearch
import pandas as pd
from pandas.core.frame import DataFrame
import time
import pymongo
import random
myclient = pymongo.MongoClient('mongodb://47.93.220.108:27017/')
mydb = myclient['movie']
classes = {
    '恐怖':1, 
    '传记':2, 
    '歌舞':3, 
    '武侠':4, 
    '戏曲':5, 
    '灾难':6, 
    '剧情':7, 
    '动画':8, 
    '喜剧':9, 
    '纪录片':10, 
    '音乐':11, 
    '同性':12, 
    '短片':13, 
    '爱情':14, 
    '科幻':15, 
    '未知':16, 
    '家庭':17, 
    '历史':18, 
    '战争':19, 
    '悬疑':20, 
    '古装':21, 
    '奇幻':22, 
    '儿童':23, 
    '犯罪':24, 
    '惊悚':25, 
    '冒险':26, 
    '运动':27, 
    '动作':28
}
es = Elasticsearch(hosts='106.13.117.37', port='9200')

body = {
    "query":{
        "match_all":{}
    }
}
data1 = pd.DataFrame([i['_source'] for i in es.search(index='moviecomment',body=body,size=13000).get('hits').get('hits')])

body = {
    "query":{
        "match_all":{}
    }
}
data2 = pd.DataFrame([i['_source'] for i in es.search(index='rankinglist',body=body,size=13000).get('hits').get('hits')])

data2.head(5)

data1.head(5)

l = []
for i in data2['production_area']:
    if i == '中国台湾' or i=='中国香港':
        i = '中国大陆'
    l.append(i)

data2['country'] = l

#comment
data1.to_csv('data1.csv')
#排行榜
data2.to_csv('data2.csv')


data1 = pd.read_csv('data1.csv')
data2 = pd.read_csv('data2.csv')
#国家量化
data_country_quantification = data2.groupby(['production_area'])['movie_officeBox'].sum().round(1)
data_country_quantification.to_csv('data_country_quantification.csv')
data_country_quantification = pd.read_csv('data_country_quantification.csv')

data_all = []
for index,row in data1.iterrows():
    #1,movie_country
    try:
        movie_country = row['movie_country'].split(' / ')[0]
    except:
        movie_country = row['movie_country']
    try:
        movie_country = float(data_country_quantification[data_country_quantification['production_area'] == movie_country]['movie_officeBox'])
    except:
        movie_country = 0
    #print(movie_country)

    #2,datetime,格林尼治时间
    movie_date = str(row['movie_date'])
    if(movie_date == '未知'):
        movie_date = 0
    else:
        try:
            movie_date = movie_date.split('(')[0]
        except:
            pass
        try:
            movie_date = int(time.mktime(time.strptime(movie_date, "%Y-%m-%d")))
        except:
            try:
                movie_date = int(time.mktime(time.strptime(movie_date, "%Y-%m")))
            except:
                movie_date = int(time.mktime(time.strptime(movie_date, "%Y")))
    #print(movie_date)

    #3,时长
    try:
        movie_length = int(row['movie_length'])
    except:
        movie_length = 0
    #print(movie_length)

    #4,导演
    movie_director = row['movie_director']
    movie_director = mydb.director.find({'name':movie_director})[0]['rank']
    #print(movie_director)

    #5,演员
    movie_star = []
    movie_star_info = row['movie_star_info'].split('movie_star_name')[1:]
    for i in movie_star_info:
        movie_star.append(i.split(' ')[1][1:])

    if(len(movie_star) == 0):
        movie_star_1 = random.randint(10,18)
        movie_star_2 = random.randint(10,18)
    else:
        try:
            movie_star_1 = mydb.actor.find({'actor':movie_star[1]})[0]['rank'] 
        except :
            movie_star_1 = random.randint(10,18)
        try:
            movie_star_2 = mydb.actor.find({'actor':movie_star[2]})[0]['rank'] 
        except :
            movie_star_2 = random.randint(10,18)
    #print(str(movie_star_1)+' '+str(movie_star_2))

    #6,类型
    movie_class = str(row['movie_class'])
    movie_class = classes[movie_class.split(',')[0]]
    #print(movie_class)

    #7,总票房
    boxOffice = 0
    movie_name = str(row['movie_name'])
    for i in mydb.movieInfo.find({'movie_name':movie_name}):
        boxOffice += float(i['boxOffice'].split('万')[0].replace('..','.'))
    boxOffice = round(boxOffice,1)
    #print(boxOffice)

    data = {
        'movie_country':movie_country,
        'movie_date':movie_date,
        'movie_length':movie_length,
        'movie_director':movie_director,
        'movie_star_1':movie_star_1,
        'movie_star_2':movie_star_2,
        'movie_class':movie_class,
        'boxOffice':boxOffice,
    }
    data_all.append(data)

data_all = pd.DataFrame(data_all)

data_all.to_csv('data_collection.csv')